package spider2

import (
	"fmt"
	log "github.com/sirupsen/logrus"
	"net/http"
)

type ConcurrentEngine struct {
	Scheduler   Scheduler
	WorkerCount int
}

type Scheduler interface {
	Submit(Request)
}

func (e ConcurrentEngine) Run(seeds ...Request) {
	for _, r := range seeds {
		e.Scheduler.Submit(r)
	}
	in := make(chan Request)
	out := make(chan ParserResult)
	for i := 0; i < e.WorkerCount; i++ {
		createWorker(in, out)
	}
	for {
		result := <-out
		for _, v := range result.Items {
			fmt.Printf("Got item:%v", v)
		}
		for _, request := range result.Requests {
			e.Scheduler.Submit(request)
		}
	}
}

func createWorker(in chan Request, out chan ParserResult) {
	go func() {
		for {
			request := <-in
			result, _ := worker(request)
			out <- result
		}
	}()
}

func worker(r Request) (ParserResult, error) {
	res, err := http.Get(r.Url)
	if nil != err {
		panic(err)
	}
	mr := UTF8Reader{R: res.Body}
	resultBytes := mr.TransReader()
	log.Info("请求的数据为%s", resultBytes)
	fmt.Printf("%p\n", r.Parser)
	request, _ := r.Parser.Parse(&resultBytes)

	result := ParserResult{}
	for _, v := range *request {
		result.Requests = append(result.Requests, *v)
	}

	return result, nil
}
